Introduce and use a per-CPU read-mostly sub-section

Since mixing data that only gets setup once and then (perhaps
frequently) gets read by remote CPUs with data that the local CPU may
modify (again, perhaps frequently) still causes undesirable cache
protocol related bus traffic, separate the former class of objects
from the latter.

These objects converted here are just picked based on their write-once
(or write-very-rarely) properties; perhaps some more adjustments may
be desirable subsequently. The primary users of the new sub-section
will result from the next patch.

Signed-off-by: Jan Beulich <jbeulich@novell.com>

diff --git a/xen/arch/ia64/xen/vhpt.c b/xen/arch/ia64/xen/vhpt.c
index f1689df0f4d7e37e3fc92ec9d79302dd01ee46c9..1872f12d093dd8b7fa2e638dcddc20e4a89d0a1d 100644 (file)
--- a/xen/arch/ia64/xen/vhpt.c
+++ b/xen/arch/ia64/xen/vhpt.c
@@ -21,8 +21,8 @@
 #include <asm/vcpumask.h>
 #include <asm/vmmu.h>
 
-DEFINE_PER_CPU (unsigned long, vhpt_paddr);
-DEFINE_PER_CPU (unsigned long, vhpt_pend);
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long, vhpt_paddr);
+DEFINE_PER_CPU_READ_MOSTLY(unsigned long, vhpt_pend);
 #ifdef CONFIG_XEN_IA64_TLBFLUSH_CLOCK
 DEFINE_PER_CPU(volatile u32, vhpt_tlbflush_timestamp);
 #endif

diff --git a/xen/arch/ia64/xen/xen.lds.S b/xen/arch/ia64/xen/xen.lds.S
index 4daf6adfb97b71d47db11794687356da3738a0c8..2fe1905f251e32676fb858dfbc738139a1e62e82 100644 (file)
--- a/xen/arch/ia64/xen/xen.lds.S
+++ b/xen/arch/ia64/xen/xen.lds.S
@@ -187,6 +187,8 @@ SECTIONS
        {
                __per_cpu_start = .;
                *(.data.percpu)
+               . = ALIGN(SMP_CACHE_BYTES);
+               *(.data.percpu.read_mostly)
                __per_cpu_end = .;
        }
   . = __phys_per_cpu_start + PERCPU_PAGE_SIZE; /* ensure percpu data fits

diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c
index 4a30a7d395d0ed7dc0c20bc70930558278ab1db7..7a53d46358e0c981863650e5ec726a523b6e8969 100644 (file)
--- a/xen/arch/x86/hvm/vmx/vmcs.c
+++ b/xen/arch/x86/hvm/vmx/vmcs.c
@@ -66,7 +66,7 @@ u32 vmx_vmexit_control __read_mostly;
 u32 vmx_vmentry_control __read_mostly;
 bool_t cpu_has_vmx_ins_outs_instr_info __read_mostly;
 
-static DEFINE_PER_CPU(struct vmcs_struct *, host_vmcs);
+static DEFINE_PER_CPU_READ_MOSTLY(struct vmcs_struct *, host_vmcs);
 static DEFINE_PER_CPU(struct vmcs_struct *, current_vmcs);
 static DEFINE_PER_CPU(struct list_head, active_vmcs_list);
 

diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 8fb5bcdd6f1a5da13f192d83cd290c87f46faa28..ec060eadfac4da0c3802a6a55eff50929d85a15b 100644 (file)
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -111,9 +111,9 @@ extern void vesa_init(void);
 extern void vesa_mtrr_init(void);
 extern void init_tmem(void);
 
-DEFINE_PER_CPU(struct desc_struct *, gdt_table) = boot_cpu_gdt_table;
+DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, gdt_table) = boot_cpu_gdt_table;
 #ifdef CONFIG_COMPAT
-DEFINE_PER_CPU(struct desc_struct *, compat_gdt_table)
+DEFINE_PER_CPU_READ_MOSTLY(struct desc_struct *, compat_gdt_table)
     = boot_cpu_compat_gdt_table;
 #endif
 

diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index a2adcfa26af9078c94237067f46681a0aff1d026..d749d268c1ef1625a8071a4e52fe68b0c222a0b7 100644 (file)
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -78,7 +78,7 @@ char opt_nmi[10] = "fatal";
 #endif
 string_param("nmi", opt_nmi);
 
-DEFINE_PER_CPU(u32, ler_msr);
+DEFINE_PER_CPU_READ_MOSTLY(u32, ler_msr);
 
 /* Master table, used by CPU0. */
 idt_entry_t idt_table[IDT_ENTRIES];

diff --git a/xen/arch/x86/x86_32/traps.c b/xen/arch/x86/x86_32/traps.c
index 406098046cf16869fd13cf159d57e5914a1f2683..c58e5e29180f86c3a0b33974e12569cbf5064871 100644 (file)
--- a/xen/arch/x86/x86_32/traps.c
+++ b/xen/arch/x86/x86_32/traps.c
@@ -188,7 +188,7 @@ void show_page_walk(unsigned long addr)
     unmap_domain_page(l1t);
 }
 
-DEFINE_PER_CPU(struct tss_struct *, doublefault_tss);
+DEFINE_PER_CPU_READ_MOSTLY(struct tss_struct *, doublefault_tss);
 static unsigned char __attribute__ ((__section__ (".bss.page_aligned")))
     boot_cpu_doublefault_space[PAGE_SIZE];
 

diff --git a/xen/arch/x86/x86_32/xen.lds.S b/xen/arch/x86/x86_32/xen.lds.S
index 1bbb65405a4db04d73d870ae970c5c2018d21578..d4afcc58a7178e067d16ef724c34757bfadefeec 100644 (file)
--- a/xen/arch/x86/x86_32/xen.lds.S
+++ b/xen/arch/x86/x86_32/xen.lds.S
@@ -4,6 +4,7 @@
  */
 
 #include <xen/config.h>
+#include <xen/cache.h>
 #include <asm/page.h>
 #include <asm/percpu.h>
 #undef ENTRY
@@ -69,9 +70,13 @@ SECTIONS
   __init_end = .;
 
   __per_cpu_shift = PERCPU_SHIFT; /* kdump assist */
-  __per_cpu_start = .;
-  .data.percpu : { *(.data.percpu) } :text
-  __per_cpu_data_end = .;
+  .data.percpu : {
+       __per_cpu_start = .;
+       *(.data.percpu)
+       . = ALIGN(SMP_CACHE_BYTES);
+       *(.data.percpu.read_mostly)
+       __per_cpu_data_end = .;
+  } :text
   . = __per_cpu_start + (NR_CPUS << PERCPU_SHIFT);
   . = ALIGN(PAGE_SIZE);
   __per_cpu_end = .;

diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c
index 4fafb904a96f2ac72416e0e2dae74ea2a170d5c7..87b8957ff5440e43959d9b8587b24c4dbc5854e8 100644 (file)
--- a/xen/arch/x86/x86_64/mm.c
+++ b/xen/arch/x86/x86_64/mm.c
@@ -37,7 +37,7 @@
 unsigned int m2p_compat_vstart = __HYPERVISOR_COMPAT_VIRT_START;
 #endif
 
-DEFINE_PER_CPU(void *, compat_arg_xlat);
+DEFINE_PER_CPU_READ_MOSTLY(void *, compat_arg_xlat);
 
 /* Top-level master (and idle-domain) page directory. */
 l4_pgentry_t __attribute__ ((__section__ (".bss.page_aligned")))

diff --git a/xen/arch/x86/x86_64/xen.lds.S b/xen/arch/x86/x86_64/xen.lds.S
index f6c37f02ccf9665ea95e0198d02d9648025d0248..8ef209841ee2656da0a9059b0bf4db29338c1bc4 100644 (file)
--- a/xen/arch/x86/x86_64/xen.lds.S
+++ b/xen/arch/x86/x86_64/xen.lds.S
@@ -2,6 +2,7 @@
 /* Modified for x86-64 Xen by Keir Fraser */
 
 #include <xen/config.h>
+#include <xen/cache.h>
 #include <asm/page.h>
 #include <asm/percpu.h>
 #undef ENTRY
@@ -67,9 +68,13 @@ SECTIONS
   __init_end = .;
 
   __per_cpu_shift = PERCPU_SHIFT; /* kdump assist */
-  __per_cpu_start = .;
-  .data.percpu : { *(.data.percpu) } :text
-  __per_cpu_data_end = .;
+  .data.percpu : {
+       __per_cpu_start = .;
+       *(.data.percpu)
+       . = ALIGN(SMP_CACHE_BYTES);
+       *(.data.percpu.read_mostly)
+       __per_cpu_data_end = .;
+  } :text
   . = __per_cpu_start + (NR_CPUS << PERCPU_SHIFT);
   . = ALIGN(PAGE_SIZE);
   __per_cpu_end = .;

diff --git a/xen/common/kexec.c b/xen/common/kexec.c
index baa4a657e96eec0f6a6c61e9d2db583b96150f3c..bb9fdf90a72e60acc3875ba11d69e2258ae409d8 100644 (file)
--- a/xen/common/kexec.c
+++ b/xen/common/kexec.c
@@ -27,7 +27,7 @@
 #include <compat/kexec.h>
 #endif
 
-static DEFINE_PER_CPU(void *, crash_notes);
+static DEFINE_PER_CPU_READ_MOSTLY(void *, crash_notes);
 
 static Elf_Note *xen_crash_note;
 

diff --git a/xen/common/tmem_xen.c b/xen/common/tmem_xen.c
index 9911e802ed9a3cc9458b79f6e6ec1d919009581a..f1b94239f4de1924522a8670cea797738318a82d 100644 (file)
--- a/xen/common/tmem_xen.c
+++ b/xen/common/tmem_xen.c
@@ -36,8 +36,8 @@ DECL_CYC_COUNTER(pg_copy);
  * allocated iff opt_tmem_compress */
 #define LZO_WORKMEM_BYTES LZO1X_1_MEM_COMPRESS
 #define LZO_DSTMEM_PAGES 2
-static DEFINE_PER_CPU(unsigned char *, workmem);
-static DEFINE_PER_CPU(unsigned char *, dstmem);
+static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, workmem);
+static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, dstmem);
 
 #ifdef COMPARE_COPY_PAGE_SSE2
 #include <asm/flushtlb.h>  /* REMOVE ME AFTER TEST */

diff --git a/xen/common/trace.c b/xen/common/trace.c
index 38113d21df994ce3875e67b6623a869c3f130680..594a3d68a060ad62bb1ad2badcce47beb2d97257 100644 (file)
--- a/xen/common/trace.c
+++ b/xen/common/trace.c
@@ -46,8 +46,8 @@ static unsigned int opt_tbuf_size = 0;
 integer_param("tbuf_size", opt_tbuf_size);
 
 /* Pointers to the meta-data objects for all system trace buffers */
-static DEFINE_PER_CPU(struct t_buf *, t_bufs);
-static DEFINE_PER_CPU(unsigned char *, t_data);
+static DEFINE_PER_CPU_READ_MOSTLY(struct t_buf *, t_bufs);
+static DEFINE_PER_CPU_READ_MOSTLY(unsigned char *, t_data);
 static int data_size;
 
 /* High water mark for trace buffers; */

diff --git a/xen/include/asm-ia64/linux-xen/asm/percpu.h b/xen/include/asm-ia64/linux-xen/asm/percpu.h
index 8642afe79214543a5a78c329620d70587d3e3c58..3bd030327c95ccf57155ad7d58ba965781e3d10b 100644 (file)
--- a/xen/include/asm-ia64/linux-xen/asm/percpu.h
+++ b/xen/include/asm-ia64/linux-xen/asm/percpu.h
@@ -26,9 +26,9 @@
        extern __SMALL_ADDR_AREA __typeof__(type) per_cpu__##name
 
 /* Separate out the type, so (int[3], foo) works. */
-#define DEFINE_PER_CPU(type, name)                             \
-       __attribute__((__section__(".data.percpu")))            \
-       __SMALL_ADDR_AREA __typeof__(type) per_cpu__##name
+#define __DEFINE_PER_CPU(type, name, suffix)                           \
+       __attribute__((__section__(".data.percpu" #suffix)))            \
+       __SMALL_ADDR_AREA __typeof__(type) per_cpu_##name
 
 /*
  * Pretty much a literal copy of asm-generic/percpu.h, except that percpu_modcopy() is an

diff --git a/xen/include/asm-x86/percpu.h b/xen/include/asm-x86/percpu.h
index d76206587f6c68797c8a415c32e174eb5aed54f1..ad5c667ace3b62a41d71b62e07769a1013aebe7c 100644 (file)
--- a/xen/include/asm-x86/percpu.h
+++ b/xen/include/asm-x86/percpu.h
@@ -5,9 +5,9 @@
 #define PERCPU_SIZE  (1UL << PERCPU_SHIFT)
 
 /* Separate out the type, so (int[3], foo) works. */
-#define DEFINE_PER_CPU(type, name)                      \
-    __attribute__((__section__(".data.percpu")))        \
-    __typeof__(type) per_cpu__##name
+#define __DEFINE_PER_CPU(type, name, suffix)                    \
+    __attribute__((__section__(".data.percpu" #suffix)))        \
+    __typeof__(type) per_cpu_##name
 
 /* var is in discarded region: offset to particular copy we want */
 #define per_cpu(var, cpu)  \

diff --git a/xen/include/xen/percpu.h b/xen/include/xen/percpu.h
index 34635892a538bebe56e469cbbc3db6358255f15c..9366d18750f7dd915ce24c265ccb18a6747c326a 100644 (file)
--- a/xen/include/xen/percpu.h
+++ b/xen/include/xen/percpu.h
@@ -4,6 +4,16 @@
 #include <xen/config.h>
 #include <asm/percpu.h>
 
+/*
+ * Separate out the type, so (int[3], foo) works.
+ *
+ * The _##name concatenation is being used here to prevent 'name' from getting
+ * macro expanded, while still allowing a per-architecture symbol name prefix.
+ */
+#define DEFINE_PER_CPU(type, name) __DEFINE_PER_CPU(type, _##name, )
+#define DEFINE_PER_CPU_READ_MOSTLY(type, name) \
+       __DEFINE_PER_CPU(type, _##name, .read_mostly)
+
 /* Preferred on Xen. Also see arch-defined per_cpu(). */
 #define this_cpu(var)    __get_cpu_var(var)